package com.cjh;

import java.util.List;

import org.apache.commons.lang.StringUtils;

import com.cjh.entity.HtmlPage;
import com.cjh.service.HbaseStoreService;
import com.cjh.service.PageDownloadService;
import com.cjh.service.ProcessService;
import com.dajiangtai.djt_spider.util.RedisUtil;
import com.dajiangtai.djt_spider.util.ThreadUtil;

public class StartYouku {
	private static final String URLSKEY="www.soku.com";
	private RedisUtil redisUtil=new RedisUtil();
	private ProcessService processService;
	private PageDownloadService downloadService;
	private HbaseStoreService storeService;
	public static void main(String[] args) {
		StartYouku job=new StartYouku();
		job.setDownloadService(new PageDownloadService());
		job.setProcessService(new ProcessService());
//		job.setStoreService(new HbaseStoreService());
		String url="http://www.soku.com/channel/teleplaylist_____1.html";
		job.redisUtil.add(URLSKEY, url);
		job.urlProducer();
//		job.urlConsumer();
		
	}
	
	private void urlProducer(){
		while(true){
			String url=redisUtil.poll(URLSKEY);
			if(StringUtils.isNotBlank(url)){
				HtmlPage page=downloadService.down(url);
				processService.process(page);
				List<String> urllist=page.getUrList();
				for (String each : urllist) {
					if(each.startsWith("http://www.soku.com/detail/")){
						redisUtil.add(URLSKEY, each);
						System.out.println("level2 add:::"+each);
					}else{
						System.out.println("level1 add:::"+each);
						redisUtil.add(URLSKEY, each);
					}
				}
				
				if(url.startsWith("http://www.soku.com/detail/show")){
					System.out.println("存储。。。。");
				}
				ThreadUtil.sleep(2000);
			}else{
				System.out.println("没有取到数据。。。等等。。。");
				ThreadUtil.sleep(3000);
			}
		}
	}
	
	public ProcessService getProcessService() {
		return processService;
	}
	public void setProcessService(ProcessService processService) {
		this.processService = processService;
	}
	public PageDownloadService getDownloadService() {
		return downloadService;
	}
	public void setDownloadService(PageDownloadService downloadService) {
		this.downloadService = downloadService;
	}

	public HbaseStoreService getStoreService() {
		return storeService;
	}

	public void setStoreService(HbaseStoreService storeService) {
		this.storeService = storeService;
	}
	
	
	
}
